{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7.13 |Anaconda custom (x86_64)| (default, Dec 20 2016, 23:05:08) \n",
      "[GCC 4.2.1 Compatible Apple LLVM 6.0 (clang-600.0.57)]\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from detail import Detail\n",
    "import numpy as np\n",
    "import skimage.io as io\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import pylab\n",
    "import json\n",
    "import random\n",
    "pylab.rcParams['figure.figsize'] = (10.0, 8.0)\n",
    "# print python version:\n",
    "import sys \n",
    "print(sys.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# annFile='../../json/PASCAL_trainval_merged.json' # annotations - out of date\n",
    "annFile='../../json/trainval_withkeypoints.json'\n",
    "imgDir='../../VOCdevkit/VOC2010/JPEGImages' # jpeg images\n",
    "phase='trainval'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "JSON root keys:[u'info', u'annos_occlusion', u'annos_boundary', u'annos_joints', u'annos_segmentation', u'parts', u'images', u'categories']\n",
      "Done (t=12.72s)\n",
      "creating index...\n",
      "index created! (t=5.68s)\n",
      "\n",
      "***info***\n",
      "contributor: https://sites.google.com/view/pasd\n",
      "date_created: 2017-07-18\n",
      "version: 3.0\n",
      "description: This is the 3.0 version of the CVPR 2017 PASCAL in Detail dataset.\n",
      "year: 2017\n"
     ]
    }
   ],
   "source": [
    "# initialize detail api for instance annotations\n",
    "detailsGt = Detail(annFile, imgDir, phase)\n",
    "\n",
    "print('\\n***info***')\n",
    "detailsGt.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load eval module for keypoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total category #1, total img #3547 \n"
     ]
    }
   ],
   "source": [
    "from detail import detaileval_kpt # work with python2, fail with python3\n",
    "detaileval = detaileval_kpt.DetailEvalKpt(detailsGt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fake a result json for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_imgs_human = detailsGt.getImgs(cats=['person']);\n",
    "detailRes = []\n",
    "count = 0\n",
    "for gt_img in gt_imgs_human:\n",
    "    '''\n",
    "    if(2 in gt_img['categories']):\n",
    "        detailRes[-1]['category_score'][0] = 1\n",
    "    if(23 in gt_img['categories'] and count < 200):\n",
    "        count += 1\n",
    "        detailRes[-1]['category_score'][1] = 1\n",
    "    '''\n",
    "    for k in gt_img['keypoints']:\n",
    "        detailRes.append(\n",
    "            {'image_id': gt_img['image_id'], \n",
    "             'score': 1, \n",
    "             'category_id': 284,\n",
    "             # randomly generate ketpoint in range 0-400 for testing\n",
    "             'keypoints': np.ndarray.tolist(400*np.random.rand(42))})\n",
    "        skeleton_id = k\n",
    "        # load groundtruth to get AP 1\n",
    "        # detailRes[-1]['keypoints'] = detailsGt.getKptAnno(int(skeleton_id))['keypoints'] # k['keypoints']\n",
    "    \n",
    "result = {}\n",
    "result['pred'] = detailRes\n",
    "detail_res_file = '../../res/detail_kpt_fake_result.json'\n",
    "json.dump(result, open(detail_res_file, 'w'), indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total category #1, total img #3547 \n"
     ]
    }
   ],
   "source": [
    "##### start evaluation\n",
    "from detail import detaileval_kpt\n",
    "detaileval = detaileval_kpt.DetailEvalKpt(detailsGt)\n",
    "detailRes = json.load(open('../../res/detail_kpt_fake_result.json', 'r'))['pred']\n",
    "detaileval.loadRes(detailRes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE (t=2.49s).\n"
     ]
    }
   ],
   "source": [
    "detaileval.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accumulating evaluation results...\n",
      "DONE (t=0.11s).\n"
     ]
    }
   ],
   "source": [
    "detaileval.accumulate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets= 20 ] = 0.471\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets= 20 ] = 0.837\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets= 20 ] = 0.504\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.017\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.480\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 20 ] = 0.595\n",
      " Average Recall     (AR) @[ IoU=0.50      | area=   all | maxDets= 20 ] = 0.909\n",
      " Average Recall     (AR) @[ IoU=0.75      | area=   all | maxDets= 20 ] = 0.694\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.607\n"
     ]
    }
   ],
   "source": [
    "detaileval.summarize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
